{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Object detection demo\n",
    "This notebook shows an example of object detection of an image.\n",
    "The network that is used for inference is a variant of Tiny-Yolo, whose topology is illustrated in the following picture.\n",
    "The pynq colored layers have been quantized with 1 bit for weights and 3 bit for activations, and will be executed in the HW accelerator, while the other layers are executed in python.\n",
    "\n",
    "The image processing is performed within darknet by using python bindings.\n",
    "\n",
    "\n",
    "![TinierYolo topology](Tinier-YOLO-topology.svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "require(['notebook/js/codecell'], function(codecell) {\n",
       "  codecell.CodeCell.options_default.highlight_modes[\n",
       "      'magic_text/x-csrc'] = {'reg':[/^%%microblaze/]};\n",
       "  Jupyter.notebook.events.one('kernel_ready.Kernel', function(){\n",
       "      Jupyter.notebook.get_cells().map(function(cell){\n",
       "          if (cell.cell_type == 'code'){ cell.auto_highlight(); } }) ;\n",
       "  });\n",
       "});\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sys\n",
    "import os, platform\n",
    "import json\n",
    "import numpy as np\n",
    "import cv2\n",
    "import ctypes\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image\n",
    "from datetime import datetime\n",
    "from qnn import TinierYolo\n",
    "from qnn import utils \n",
    "sys.path.append(\"/opt/darknet/python/\")\n",
    "from darknet import *\n",
    "%matplotlib inline\n",
    "import IPython"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Instantiate a Classifier\n",
    "Creating a classifier will automatically download the bitstream onto the device. All other initializations are currently performed in the Darknet framework. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = TinierYolo()\n",
    "classifier.init_accelerator()\n",
    "net = classifier.load_network(json_layer=\"/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-layers.json\")\n",
    "\n",
    "conv0_weights = np.load('/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-conv0-W.npy', encoding=\"latin1\")\n",
    "conv0_weights_correct = np.transpose(conv0_weights, axes=(3, 2, 1, 0))\n",
    "conv8_weights = np.load('/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-conv8-W.npy', encoding=\"latin1\")\n",
    "conv8_weights_correct = np.transpose(conv8_weights, axes=(3, 2, 1, 0))\n",
    "conv0_bias = np.load('/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-conv0-bias.npy', encoding=\"latin1\")\n",
    "conv0_bias_broadcast = np.broadcast_to(conv0_bias[:,np.newaxis], (net['conv1']['input'][0],net['conv1']['input'][1]*net['conv1']['input'][1]))\n",
    "conv8_bias = np.load('/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-conv8-bias.npy', encoding=\"latin1\")\n",
    "conv8_bias_broadcast = np.broadcast_to(conv8_bias[:,np.newaxis], (125,13*13))\n",
    "\n",
    "file_name_cfg = c_char_p(\"/opt/python3.6/lib/python3.6/site-packages/qnn/params/tinier-yolo-bwn-3bit-relu-nomaxpool.cfg\".encode())\n",
    "\n",
    "net_darknet = lib.parse_network_cfg(file_name_cfg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Launch demo\n",
    "\n",
    "The loop will automatically pick a random image from the yoloimages folder and perform the whole classification. Use the \"interrupt kernel\" button on top to stop the demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "out_dim = net['conv7']['output'][1]\n",
    "out_ch = net['conv7']['output'][0]\n",
    "img_folder = './yoloimages/'\n",
    "file_name_out = c_char_p(\"/home/xilinx/jupyter_notebooks/qnn/detection\".encode())\n",
    "file_name_probs = c_char_p(\"/home/xilinx/jupyter_notebooks/qnn/probabilities.txt\".encode())\n",
    "file_names_voc = c_char_p(\"/opt/darknet/data/voc.names\".encode())\n",
    "tresh = c_float(0.3)\n",
    "tresh_hier = c_float(0.5)\n",
    "darknet_path = c_char_p(\"/opt/darknet/\".encode())\n",
    "\n",
    "conv_output = classifier.get_accel_buffer(out_ch, out_dim)\n",
    "\n",
    "while(1):\n",
    "    for image_name in os.listdir(img_folder):\n",
    "        img_file = os.path.join(img_folder, image_name)\n",
    "        file_name = c_char_p(img_file.encode())\n",
    "\n",
    "        img = load_image(file_name,0,0)\n",
    "        img_letterbox = letterbox_image(img,416,416)\n",
    "        img_copy = np.copy(np.ctypeslib.as_array(img_letterbox.data, (3,416,416)))\n",
    "        img_copy = np.swapaxes(img_copy, 0,2)\n",
    "        free_image(img)\n",
    "        free_image(img_letterbox)\n",
    "\n",
    "        #First convolution layer in sw\n",
    "        if len(img_copy.shape)<4:\n",
    "            img_copy = img_copy[np.newaxis, :, :, :]\n",
    "\n",
    "        conv0_ouput = utils.conv_layer(img_copy,conv0_weights_correct,b=conv0_bias_broadcast,stride=2,padding=1)\n",
    "        conv0_output_quant = conv0_ouput.clip(0.0,4.0)\n",
    "        conv0_output_quant = utils.quantize(conv0_output_quant/4,3)\n",
    "\n",
    "        #Offload to hardware\n",
    "        conv_input = classifier.prepare_buffer(conv0_output_quant*7);\n",
    "        classifier.inference(conv_input, conv_output)\n",
    "        conv7_out = classifier.postprocess_buffer(conv_output)\n",
    "\n",
    "        #Last convolution layer in sw\n",
    "        conv7_out = conv7_out.reshape(out_dim,out_dim,out_ch)\n",
    "        conv7_out = np.swapaxes(conv7_out, 0, 1) # exp 1\n",
    "        if len(conv7_out.shape)<4:\n",
    "            conv7_out = conv7_out[np.newaxis, :, :, :] \n",
    "\n",
    "        conv8_output = utils.conv_layer(conv7_out,conv8_weights_correct,b=,stride=1)  \n",
    "        conv8_out = conv8_output.ctypes.data_as(ctypes.POINTER(ctypes.c_float))\n",
    "\n",
    "        #Draw detection boxes\n",
    "        lib.forward_region_layer_pointer_nolayer(net_darknet,conv8_out)\n",
    "        lib.draw_detection_python(net_darknet, file_name, tresh, tresh_hier,file_names_voc, darknet_path, file_name_out, file_name_probs);\n",
    "\n",
    "        #Display result\n",
    "        IPython.display.clear_output(1)\n",
    "        file_content = open(file_name_probs.value,\"r\").read().splitlines()\n",
    "        detections = []\n",
    "        for line in file_content[0:]:\n",
    "            name, probability = line.split(\": \")\n",
    "            detections.append((probability, name))\n",
    "        for det in sorted(detections, key=lambda tup: tup[0], reverse=True):\n",
    "            print(\"class: {}\\tprobability: {}\".format(det[1], det[0]))\n",
    "        res = Image.open(file_name_out.value.decode() + \".png\")\n",
    "        display(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reset the device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.deinit_accelerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pynq import Xlnk\n",
    "\n",
    "xlnk = Xlnk();\n",
    "xlnk.xlnk_reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
